# coding: utf-8
import socket
import threading
import SocketServer
import time
import pdb
import cPickle as pickle
import Queue
import urllib2
import os
import args
import bsddb3
import Queue
import re,urllib
import subprocess
import signal
##2012-9-3
##增加了一个函数RecvFmgCommend
 
#保存vide之后  执行ffmpeg提取一帧图片
#参数1  url   db 的key
#参数2 TrunCommand  是ffmpeg的命令行参数
#参数3 TurnsImgPath 是图片路径
 
#命令参数什么的 我就是把所有日志信息给保存上了。如果要求其他格式的话 我再改

lock=threading.Lock()
dllock=threading.Lock()
mlock=threading.Lock()
global taskf
taskf=open('offtask.txt','a')

global sp_domain_name
##sp_domain_name =['sina','sohu','qiyi','youku','tudou','other_sp']   ## read from pickle or file
##for i in sp_domain_name:
##    if os.path.exists(dbhome+'/'+i) == False:
##        os.mkdir(dbhome+'/'+i)

global shutdown
shutdown=0

global server

q=Queue.Queue()
request_q=Queue.Queue()
global df
df=open('download.txt','a')

maxthreadq=Queue.Queue()

global argscomm,argsname,argsvalue,argsdict,conffile
global offtcmgrname,offtcmgrdomain,offtcmgrip,offtcmgrport
global tcdomain,tcclusterip,tcport,tcsymbole
global cachedomain,cacheclusterip,cacheport,cacherouter
global dbhome,maxthread,manual
global log_t,log_ip,log_port,myip,myport
global _debug_


dbhome='./db_home'
_debug_=1

import sendlog
global log_t,log_ip,log_port
##class logmsg_body():
##    def __init__(self,msglist):
##create table LOG
##(
##   ID                     int not null auto_increment,
##  0 BEGINTIME              datetime comment '开始时间',
##  1 ENDTIME                timestamp default CURRENT_TIMESTAMP comment '结束时间',
##  2 LOG_LEVEL              int comment '日志级别（INFO 5，LOG 4，WARNING 3，ERROR 2，CRITICAL 1）',
##  3 LOG_TYPE               int comment '日志类型{1=TCD, 2=TC, 3=OFFTCMGR}',
##  4 HOST_DOMAIN            varchar(30) comment '服务器域名',
##  5 IP_PORT                varchar(20) comment 'IP地址+端口号',
##  6 PROCESS                varchar(20) comment '进程名称（TC，TCD，OFFMGR, SYSTEM）',
##  7 CONTENT_TYPE           varchar(20) comment '内容类型FLV mp4 3gp mkv avi ts application-oct-stream ...',
##  8 CONTENT_LEN            int comment '内容长度kB',
##  9 ORIGIN_LEN             int comment '原始长度kB',
##  10 ORIGIN_DURATION        int comment '原始持续时间，？秒',
##  11 REQUEST_TRAFFIC        int comment '本次请求实际流量，单位kB,长度为2的32次方',
##  12 REQUEST_ORIGIN_TRAFFIC int comment '本次请求原始流量，单位kB,长度为2的32次方',
##  13 TC_TYPE                int comment 'TC类型{1=ONLINE, 2=OFFLINE, 3=ORIGIN}',
##  14 TC_TYPE_NAME           varchar(20) comment 'TC 类型{1=ONLINE, 2=OFFLINE, 3=ORIGIN}',
##  15 VIDEO_TYPE             int comment '封装容器，视频类型file type｛1=MP4,2=FLV,3=3GP,……｝',
##  16 VIDEO_TYPE_NAME        varchar(20) comment '封装容器，视频类型file type｛1=MP4,2=FLV,3=3GP,……｝',
##  17 VIDEO_CODEC            int comment '视频编码器{263=H263,264=H264,4=MPEG4,……}',
##  18 VIDEO_CODEC_NAME       varchar(20) comment'视频编码器{263=H263,264=H264,4=MPEG4,……}',
##  19 URL                    varchar(222) comment 'url',
##  20 INFO                   varchar(222) comment '信息',
##  21 USER_AGENT_SIZE        int comment '手机尺寸',
##  22 USER_AGENT_SIZE_NAME   varchar(20) comment '手机尺寸',
##  23 USER_AGENT_TYPE        int comment '手机品牌',
##  24 USER_AGENT_TYPE_NAME   varchar(20) comment '手机品牌',
##  25 USER_AGENT_MODEL       int comment '手机型号，手机品牌下的子类，可以在手机品牌下进行检索',
##  26 USER_AGENT_MODEL_NAME  varchar(20) comment '手机型号，手机品牌下的子类，可以在手机品牌下进行检索',
##  27 SP_DOMAIN              int comment '网站域名',
##  28 SP_DOMAIN_NAME         varchar(20) comment '网站域名',
##  Primary key (ID)
##);
## msgdict  key:value, key 编码数字参见上述定义
def offtcmgr_logmsg_body(msgdict):
    begingtime=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
    endtime=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
    templist=[begingtime,endtime,'4','3',\
              offtcmgrname,myip+':'+str(myport),'offmgr','',\
              '0','0','0','0',\
              '0','0','' ,'0',\
               '','0','' ,'' ,\
              'ok','0','' ,'0',\
               '','0','' ,'0',\
              '']
    msgsend=''
    msgkeys=msgdict.keys()
    for i in range(len(templist)):
        if i in msgkeys: ##msglist[i] :
            if 'str'  in str(type(msgdict[i])):
                msgsend=msgsend+msgdict[i]
            else:
                msgsend=msgsend+str(msgdict[i])
        else:
            msgsend=msgsend+templist[i]
        msgsend=msgsend+','
##    msgsend=begingtime+','+endtime+','+'4,3,'+\
##             offtcmgrip+':'+str(offtcmgrport)+','+offtcmgrip+':'+str(offtcmgrport)+',offmgr,'+\
##             ',0,0,0,0,0,0,,0,,0,,,'+'offtcmgr start ok'+','+\
##             '0,,0,,0,,0,'
    return msgsend[:-1]
    
##pp=threading.Thread(target=sendlog.Sn_Thread_SendAll,args=(5,1000,  "192.168.0.104", logging.handlers.DEFAULT_TCP_LOGGING_PORT))
##pp.start()
##log_t = sendlog.SN_SendLog() #ip='192.168.0.104'
##log_t=sendlog.SN_SendLog()


global_manualDownQue = Queue.Queue() 
global global_manualDownArray
global_manualDownArray = []
global_manualDownArray=bsddb3.btopen('./manual_download.db','c')
global global_urlinfoDict
global_urlinfoDict = {}  #url转码信息字典
##class BindableHTTPConnection(http.client.HTTPConnection):
##    def connect(self):
####        
##        """Connect to the host and port specified in __init__."""
##        self.sock = socket.socket()
##        self.sock.bind((self.source_ip, 0))
##        if isinstance(self.timeout, float):
##                self.sock.settimeout(self.timeout)
##        self.sock.connect((self.host,self.port))
##
##def BindableHTTPConnectionFactory(source_ip):
##    def _get(host, port=None, strict=None, timeout=0):
##        bhc=BindableHTTPConnection(host, port=port, strict=strict, timeout=timeout)
##        bhc.source_ip=source_ip
##        return bhc
##    return _get
##
##class BindableHTTPHandler(urllib.request.HTTPHandler):
##    global bindip
##    def http_open(self, req):
##        return self.do_open(BindableHTTPConnectionFactory(bindip), req)  #

def dtconfig(tcp_self_path):
    ##urlset='/dtconfig/dtupdateargs/serverip:192.168.0.200'
    ##
    ##urlget='/dtconfig/getargs/serverip'
    global  dtconlock
##    global argscomm,argsname,argsvalue,argsdict,conffile
##    global offtcmgrip,offtcmgrport,protype
##    global tcname,tcclusterip,tcport,tcsymbole
##    global cachename,cacheclusterip,cacheport,cacherouter
##    global dbhome,maxthread

    global argscomm,argsname,argsvalue,argsdict,conffile
    global offtcmgrname,offtcmgrdomain,offtcmgrip,offtcmgrport
    global tcdomain,tcclusterip,tcport,tcsymbole
    global cachedomain,cacheclusterip,cacheport,cacherouter
    global dbhome,maxthread,manual
    global log_t,log_ip,log_port,myip,myport
    global _debug_
    
    self_path=tcp_self_path.split()[1]
##    dtconlock.acquire()
    returncode='ok'
    try:
        if 'dtupdateargs' in self_path :
##            
            argstr=(self_path.split('dtupdateargs/')[1]).split(':',1)
            argname=argstr[0]
            argvalue=argstr[1]
            try:
                globals()[argname]=args.setargsvalue(argvalue,globals()[argname])
                args.dtupdateargs(argsvalue,argsdict,argname,argvalue,'')
                args.dumpargs(argscomm,argsname,argsvalue,conffile)
            except:
                returncode='error'
        if 'getargs' in self_path :
            templist=self_path.split('getargs/')
            if self_path[-8:]=='getargs/':
                returncode=args.dumpargs_pickle(argscomm,argsname,argsvalue)
            else:
                argname=templist[1]
        ##        argname=argstr[0]
        ##        if argname in globals().keys():
        ##            returncode= globals()[argname]
                if argname in globals().keys():
                    temp= globals()[argname]
                    returncode=''
                    if 'list' in str(type(temp)):
                        for i in temp:
                            returncode=returncode+i+','
                    elif 'dict' in str(type(temp)):
                        for i in temp.keys():
                            returncode=returncode+i+':'+temp[i]+','
                    elif 'int' in str(type(temp)):
                        return str(temp)
                    else:
                        return temp
                    returncode=returncode[:-1]  ##delete ','
                else:
                    returncode='nokey'
    except:
        ##记录告警或错误：动态参数配置失败
        pass
    return returncode



class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):

    def handle(self):
##        print('offmgr')
        global lock,taskf,q,request_q
        ag=1;sums=0
        #
        data=''
        try:
            #tempdata=self.request.recv(3*1024)
            while 1:
                tempdata=self.request.recv(10240)
                data=data+tempdata
                #size=len(tempdata)
###                sums=sums+size
###                print('recv sums:',sums)
                if('\r\n\r\n' in data[-4:]): #size==0):
                    data=data[:-4]
                    print 'recv tc request:',data, \
                          time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
                    break #ag=0
                time.sleep(0.02)
        except Exception as sockerror:
            pass
        #self.request.send('HTTP/1.1 200 OK\r\nDate: Wed, 14 Apr 2012 23:23:23 GMT\r\n
        #Server: myvcff server\r\nConnection: Keep-Alive\r\nContent-Length: '+str(len(videodata))+'\r\n
        #Content-Type: video/f4v\r\n\r\n')
        #self.request.send(videodata)
        #self.request.send('\r\n')
        if('dtconfig' in data):    ##动态设置流量控制参数，不需要重启服务程序
            output=dtconfig(data)
##            time.strftime("%Y-%m-%d %H:%M:%S",time.localtime        'Wed, 14 Apr 2012 23:23:23 GMT\r\n'
            datetime=time.strftime("%a, %d %b %Y %H:%M:%S CST",time.localtime(time.time()))
            self.request.send('HTTP/1.1 200 OK\r\nDate: '+    datetime      )
            self.request.send('Server: offtcmgr server\r\nConnection: Keep-Alive\r\nContent-Length: ')
            self.request.send(str(len(output))+'\r\n')
            self.request.send('Content-Type: text/html\r\n\r\n')
            self.request.send(output)
        elif  data.split('\r\n',1)[0].strip() in 'untc tcstatus manual' :
            uiConmendThread(data,self.request)
        else:
            lock.acquire()
            taskf.write(str(data)+': '+str(time.time())+ '\r\n')
            taskf.flush()
##            print(data)
            if 'str' in str(type(data)):
                request_q.put(data)
            else:
                request_q.put(str(data))
            lock.release()
        self.request.close()
            
##        download()
##            break
##        else: print('get lock failure')
##            count=count-1
##            print('count: ',count)
##        flush_stdout()
##        lock.release()

class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
    pass

global tm_DictNum
tm_DictNum = 0
#手动下载  输入下载个数
def manualdown(count):
    global global_manualDownQue
    #校验参数
    if count < 1:
        print ("error:manualdown count < 1")
        return
    
    tempcount = 0
    global shutdown
    while global_manualDownQue.qsize() >0:
        if shutdown:break
        item = global_manualDownQue.get()
        prf=threading.Thread(target=download,args=(item,1))
        prf.start()
        tempcount = tempcount+1
        if tempcount > count:
            break

#保存vide之后  执行ffmpeg提取一帧图片
#参数1  url   db 的key
#参数2 TrunCommand  是ffmpeg的命令行参数 ffmpeg -i  TurnCommand vfilename.jpg
#参数3 vfilename 完整的视频文件路径
def RecvFmgCommend(url="http://VideoSave" , TurnCommand="-ss 3 -t 0.001 -f image2", vfilename = "ski.mp4"):
    #从视频路径中获取jpg名字
    impName = vfilename[(vfilename.rfind('/')+1):] + ".jpg"
    sntime = '%17.6f'%time.time()
    command = + "ffmpeg -i "+ vfilename +TurnCommand + impName #完整的命令行
    sub = subprocess.Popen("nohup " + command +" >"+ sntime +".txt 2>&1 &", 
                           shell=True, close_fds=True, # 必须加上close_fds=True，否则子进程会一直存在
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
    
    vdimglog=open(sntime + ".txt", "r")
    sCommandLog = vdimglog.read()
    vdimglog.close()
    os.remove(sntime + ".txt")
    
    #根据video的来源，提取出的img  保存到 db中
    vdimgPath_bdb=bsddb3.hashopen(dbhome+'vdimgPath.db','c')
    vdimgPath_bdb[url] = impName  ##俞  需要改变用 download里面的$filename.jpg
    vdimgPath_bdb.close()
    
    #根据video的来源 ，执行ffmpeg命令的返回日志 保存到 db中
    vdimglog_bdb=bsddb3.hashopen(dbhome+'vdimglog.db','c')
    vdimglog_bdb[url] = sCommandLog
    vdimglog_bdb.close()
    
    #分割log 
    videoattribute_bdb=bsddb3.hashopen(dbhome+'videoattribute.db','c')
    vdict = {}
    vdict["Command line"] = command
    
    nInput = sCommandLog.find("Input")
    temps = sCommandLog[nInput+6:sCommandLog.find(":", nInput)]
    vdict["Input"] = temps
    
    nDuration = sCommandLog.find("Duration:")
    temps = sCommandLog[nDuration+10:sCommandLog.find(",", nDuration)]
    vdict["Duration"]=temps
    
    nstart = sCommandLog.find("start:")
    temps = sCommandLog[nstart+7:sCommandLog.find(",", nstart)]
    vdict["start"]=temps
    
    nbitrate = sCommandLog.find("bitrate:")
    temps = sCommandLog[nbitrate+9:sCommandLog.find(",", nbitrate)]
    vdict["bitrate"]=temps
    
    nVideo = sCommandLog.find("Video:")
    temps = sCommandLog[nVideo+7:sCommandLog.find("\r", nVideo)]
    tempList = temps.split(",")
    vdict["Video"]=tempList
    
    nAudio = sCommandLog.find("Audio:")
    temps = sCommandLog[nAudio+7:sCommandLog.find("\r", nAudio)]
    tempList = temps.split(",")
    vdict["Audio"]=tempList
    
    nbuffer = sCommandLog.find("buffer @")
    tempdict = {}
    tempdict["w"]=sCommandLog[sCommandLog.find("w:", nbuffer)+2:sCommandLog.find("h:", nbuffer)]
    tempdict["h"]=sCommandLog[sCommandLog.find("h:", nbuffer)+2:sCommandLog.find("pixfmt:", nbuffer)]
    tempdict["pixfmt"]=sCommandLog[sCommandLog.find("pixfmt:", nbuffer)+7:sCommandLog.find("\r", nbuffer)]
    vdict["buffer"]=tempdict
    
    nscale = sCommandLog.find("scale @")
    tempdict = {}
    tempdict["w"]=sCommandLog[sCommandLog.find("w:", nscale)+2:sCommandLog.find("h:", nscale)]
    tempdict["h"]=sCommandLog[sCommandLog.find("h:", nscale)+2:sCommandLog.find("pixfmt:", nscale)]
    tempdict["fmt"]=sCommandLog[sCommandLog.find("fmt:", nscale)+4:sCommandLog.find("\r", nscale)]
    vdict["scale"]=tempdict
    
    videoattribute_bdb[url]=pickle.dumps(vdict)
    videoattribute_bdb.close()


def download(url,i):
##url format:
## tcargcount==2  http://v.sohu.com/1.mp4
##&&&tc&&&-s&320x240&-vcodec&libx264&-r&10&-b:v&100000
## tcargcount==3
##http://10.47.158.87:8000/
##&&&tc&&&-s&320x240&-vcodec&libx264&-r&10&-b:v&100000
##&&&tc&&&http://v.sohu.com/1.mp4
    if _debug_ :
        df.write(url+'\r\n')
        df.flush()
        #return
    ## 下面代码没有测试过
##    global df
##    item=q.get()
    #dllock.acquire()
    global proxyip  #从配置文件中读取
    global proxyheaders  #需要从数据库中读取,测试的时候填：user-agent,host,基本差不多
    proxyip = cacheclusterip[0]+':'+str(cacheport)
##    proxyheaders = {'user-agent':'offtcmgr','host':'offtcmgr','TCMode':'offline'}
    proxyheaders=[('user-agent','offmgr'),('TCMode','offline')] ##
    proxy_handler=urllib2.ProxyHandler({'http': proxyip})
    bindip=''
    if(proxyip):  #从指定的ip地址（网卡）出
        if(bindip):
            opener = urllib2.build_opener(proxy_handler,BindableHTTPHandler)
        else:
            opener = urllib2.build_opener(proxy_handler)
        opener.addheaders = proxyheaders
        response = opener.open(url)
##        ('GET http://192.168.0.104:7900/&&&tc&&&-s&640x480&-f&flv&&&tc&&&http://192.168.0.104:8020/out000_qt.mp4 HTTP/1.1\r\n
##         Accept-Encoding: identity\r\n
##         Host: 192.168.0.104:7900\r\nConnection: close\r\nUser-Agent: Python-urllib/2.6', '\r\n')
    else:
        req=urllib2.Request(url)
        req.addheaders=proxyheaders
        #req.headers=proxyheaders
        response=urllib2.urlopen(req)
    http_header='%s'%response.info()
    proxy_respheader= dict(re.findall(r"(?P<name>.*?): (?P<value>.*?)\r\n", http_header))
    print 'open url, and response header:\r\n',proxy_respheader,\
          time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
##    (Pdb) print proxy_respheader
##Server: BaseHTTP/0.6 Python/3.2.2
##Date: Thu, 06 Sep 2012 22:35:19 GMT
##Date: Wed, 14 Apr 2012 23:23:23 GMT
##Server: myvcff server
##Connection: Keep-Alive
##Content-Type: video/
##Transfer-Encoding: chunked

    f_size=0
    chunked=0
##    
    if('Content-Length' in proxy_respheader):
        f_size=int(proxy_respheader['Content-Length'])
    if ('Transfer-Encoding: chunked' in proxy_respheader):
        chunked=1
    sizestatus=f_size
    #分析url格式
    global sp_domain_name   #=['sina','sohu','qiyi','youku','tudou']   ## read from pickle or file
##    if not sp_domain_name:
##        proto, rest = urllib2.splittype(url)  
##        hostaddr, rest = urllib2.splithost(rest)  
##        
##        hostaddr = hostaddr.replace("\r", "")
##        hostaddr = hostaddr.replace("\n", "")
##        if os.path.exists(hostaddr) == False:
##            os.mkdir(hostaddr)
##            #文件夹保存在数据库中
##            dbmgrurl = bsddb3.hashopen('dbmgrurl.db','c')
##            dbmgrurl[url] = hostaddr
##            dbmgrurl.close()
##        filename=hash(rest)
##        f=open(hostaddr+"\\"+str(filename)+".txt",'wb')
####    filename=hash(url)
####    f=open(str(filename)+".txt",'wb')
##        urldb=bsddb3.hashopen(dbhome+'url.db','c')
##    ##需要从url提取 域名，作一个分类，需补充完成
##        domain = "22.33.22"
##        urldb[url]='%10s'%domain+str(filename)
##        urldb.close()
##        uheaddb=bsddb3.hashopen(dbhome+'uhead.db','c')
##        uheaddb[url]=str(proxy_respheader)
##        uheaddb.close()
    filename=''
    for i in sp_domain_name:  #sp_domain_name，已经通过参数文件读入，
        #是一个list，如：sina、sohu、qiyi，每个域名一个目录文件夹，不在list中的域名，进other_sp目录
        if i in url:
            filename=dbhome+'/'+i+'/'+str(hash(url))
    if not filename:
        filename=dbhome+'/'+'other_sp'+'/'+str(hash(url))
        
    #保存 filename的值
    offtc_file_bdb =bsddb3.hashopen('./db_home/offtc_file.db','c')
    offtc_file_bdb[url] = fileName
    offtc_file_bdb.close()
    
    f=open(filename,'wb')
    global tm_DictNum
    dllock.acquire()
    tm_DictNum +=1
    cur_DictNum = tm_DictNum
    global_urlinfoDict[cur_DictNum] = url + "&OFF&" + "视频名称" +"&OFF&" + str(f_size)+"&OFF&0\r\n"
    dllock.release()
    global shutdown
    if chunked:
        sizestatus=1024000000
    count=8
    recvsize=0
    while sizestatus>0:
        data=response.read(102400)
        if(data):
            f.write(data)
            count=8
        else:
            count=count-1
            if count<=0 :
                print 'count =0, maybe recv ok ',\
                      time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
                break        
        sizestatus=sizestatus-len(data)
        recvsize=recvsize+len(data)
        if sizestatus<=0:
            print 'recv ok  sizestatus =',sizestatus,\
                  time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
            urltcdonedb=bsddb3.hashopen('./db_home/urltcdone.db','c')
            urltcdonedb[url]='done'
            urltcdonedb.close()
            del global_manualDownArray[url]
            global_manualDownArray.sync()
        dllock.acquire()
        global_urlinfoDict[cur_DictNum] = url + "&OFF&" + "视频名称" +"&OFF&" + str(f_size)+\
            "&OFF&"+ str(recvsize)+"\r\n"
        dllock.release()
        if shutdown:break
        #将sizestatus发送给界面显示进度，此功能需补充完成
    #if('Content-Type' in proxy_respheader  and \
    f.close()
    response.close()
    df.write(url+'\r\n')
    df.flush()
    #dllock.release()
    maxthreadq.get()
    
    #转码后  提取图片
    TrunCommand = "-ss 3 -t 0.001 -f image2"
    RecvFmgCommend(url, TrunCommand, filename)
    
    #转码完成后从字典中删除
##    del global_urlinfoDict[cur_DictNum]
    ##需要更新tcd的数据库，update urltcdonedb

shutdown=0
def workth(ip,port):
    global q,request_q,df,shutdown
    global maxthread,maxthreadq,manual
##    manual = 0
    while True:
        checkmaxthread=1
        if shutdown:break
        try:
            while 1:
                if request_q.empty():
                    break
                else:
                    item=request_q.get(timeout=3)
            #是否是手动
                    if manual:
                        #把下载地址加进数组
##                        mlock.acquire()
                        global_manualDownArray[item]='0'  ## 0: 放入待下载队列中，但是没有启动，需要手工启动下载转码任务
##                        global_manualDownArray.append(item)
##                        mlock.release()
                    else:
##                        mlock.acquire()
                        q.put(item)
                        global_manualDownArray[item]='1'  ## 1：已经提交下载转码任务队列中
##                        global_manualDownArray.append(item)
##                        mlock.release()
                    print ('has put to global_manualDownArray or q')
            if item:
                global_manualDownArray.sync()
                        
        except Exception as ex:
            pass
        try:
            item=q.get(timeout=3)
            prf=threading.Thread(target=download,args=(item,1))
            prf.start()
            maxthreadq.put(prf)
        except Queue.Empty as ex:
            if shutdown:break
        while checkmaxthread:
            if shutdown:break
            if maxthreadq.qsize()>=maxthread:
                time.sleep(3)
            else:
                checkmaxthread=0

def client(ip, port, message):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((ip, port))
    sock.send(message+'\r\n\r\n')
    #while True:
        #response = sock.recv(4096)
        #if(len(response)==0):
            #break
##    print("Received: %s" % response)
    sock.close()

#界面Tcp线程采集
def uiConmendThread(tcpdata, connection):
    global q
##    ADDR = (ip, port)   #地址
##
##    sockobj = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
##    sockobj.bind(ADDR)
##    sockobj.listen(5)
##    
##    while True:
##        print 'waiting for connecting...'
##        connection, address = sockobj.accept()
##        print 'connected from:', address
##        
        
    while True:
        try:
            if (not tcpdata):
                connection.send("commend error")
            elif tcpdata == "untc":#返回未转码的视频的URL的list字符串
                strsend = "\r\n"
                mlock.acquire()
##                i=len(global_manualDownArray)
##                for undownurl in global_manualDownArray:
##                    strsend +=  undownurl +"\r\n"
####                strsend='untc response data\r\n\r\n'
                keys=global_manualDownArray.keys()
                mlock.release()
                for undownurl in keys:
                    strsend +=  undownurl +"\r\n"
                connection.send(strsend)
                print('untc ',strsend)
                
            elif tcpdata == 'tcstatus':#转码进度
                strsend = "\r\n"
                dllock.acquire()
                if len(global_urlinfoDict)>0:
                    keys=global_urlinfoDict.keys()
                    for i in keys:
                        strsend +=global_urlinfoDict[i]
##                    dllock.release()
##                strsend='http://192.168.0.100:8020/out000_qt.mp4' + "&OFF&" + "视频名称" +"&OFF&" + '250000'+"&OFF&0\r\n"
                print 'query tcstatus ,response data:',strsend
                connection.send(strsend)
####                
                if len(global_urlinfoDict)>0:  ##查询过后都删除，由程序其它地方再更新
##                    dllock.acquire()
                    for i in keys:     
                        del global_urlinfoDict[i]
                dllock.release()
            elif tcpdata[:3] == 'add':#添加转码url list
                urllist = tcpdata[5:].split("\r\n")
                mlock.acquire()
                for i in urllist:
                    if i:
                        global_manualDownArray[i]='0'
                if urllist:
                    global_manualDownArray.sync()
                mlock.release()
            elif tcpdata[:6] == 'manual':#手动启动未转码url list
                urllist = tcpdata[8:].split("\r\n")
                mlock.acquire()
##                for i in urllist:
##                    if i:
##                        global_manualDownArray[i]='0'
                #把 这条记录加入下载队列 q 中，稍后从手动转码地址列表中删除
##                mql=len(global_manualDownArray)
##                i=0
##                while 1:
##                    if i >=len(global_manualDownArray):
##                        break
##                    else:
##                        q.put(global_manualDownArray[i])
##                        del global_manualDownArray[i]
####                for j in range(len(global_manualDownArray)):
####                    
####                    if global_manualDownArray[i] == url:
####                        del global_manualDownArray[j]
####                        break
####                        #启动线程下载
####                        prf=threading.Thread(target=download,args=(i,1))
####                        prf.start()

                
                keys=global_manualDownArray.keys()
                mlock.release()
                for i  in keys:
                    if global_manualDownArray[i]=='0':
                        q.put(i)
                        global_manualDownArray[i]='1'
                connection.send("has appended the download queue")
                print 'manual ok'
                if keys:
                    global_manualDownArray.sync()
##                connection.close()
##                break
            if shutdown:break
##            
            tcpdata = connection.recv(1024)
            if len(tcpdata)>=4 and tcpdata[-4:]=='\r\n\r\n':
                tcpdata=tcpdata[:-4]
        except:
            break
    connection.close()
##    sockobj.close()

def tempthread(ip, ht):
    time.sleep(10)
    manualdown(2)


def offtcmgrshutdown(signalNun,   currentStackFrame):
##    timetemp0=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))
##    msgsend='1'+timetemp0+','+timetemp0+','+'4,2,192.168.0.1:8000,192.168.0.1:8000,tcd,,'\
##                 ',,,,,,,,,tcd server shut down,'
##    log_t.info(msgsend)
    taskf.close()
    df.close()
    global server
    server.socket.close()
##    msgdict={20:'offtcmgr shut down ok'}
##    msgsend=offtcmgr_logmsg_body(msgdict)
##    log_t.error('1'+msgsend)
##    sendlog.sendlogDown()
##    global shutdown
##    shutdown=1
##    try:
##        os.remove('offpidfile')
##    except:pass
##    time.sleep(3)
##    sys.exit()




def main():
##if __name__ == "__main__":
    # Port 0 means to select an arbitrary unused port
    #
    #HOST, PORT = "localhost", 8000

    #server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler)
    #ip, port = server.server_address
    #server.request_queue_size=500

## Start a thread with the server -- that thread will then start one
## more thread for each request
    #server_thread = threading.Thread(target=server.serve_forever)
## Exit the server thread when the main thread terminates
###    server_thread.setDaemon(True)
    #server_thread.start()
    #print("Server loop running in thread:", server_thread.name)
    #time.sleep(1)

##client(ip, port, b"Hello World 1")
##client(ip, port, b"Hello World 2")
##client(ip, port, b"Hello World 3")
##client(ip, port, b"Hello World 4")
    #if _debug_:
        #import bsddb3
        #urltcdone=bsddb3.hashopen(dbhome+'urltcdone.db','c')
        #for i in urltcdone.keys():
            #del urltcdone[i]
        #urltcdone.close()
    #begintime=time.time()
    #message='GET /1.flv HTTP/1.1\r\nHost: myvcff.net\r\nUser-Agent: Python vcff\r\n\r\n'
    #port=8000
    #for i in range(10):
        #prf=threading.Thread(target=client,args=(ip,port,message))
        #prf.start()
##    time.sleep(1)
##    for i in range(1000):
##        prf=threading.Thread(target=client,args=(ip,port,b'http://news.sina.com.cn/'))
##        prf.start()    
    #print(time.time()-begintime)
    #input('any key will shutdown server')
    #server.shutdown()

    #以下是加密授权部分 先去掉
##    log_t
    import mac
    try:
        miwenf=open('license','rb')
        data=miwenf.read()
        miwenf.close()
        if not mac.de_mac(data):
            print('no license !')  ## will redirect to offmgrmix8010.py.out
##            msgsend=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())+','+\
##                     time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())+','+\
##                     '4,3,'+offtcmgrip+':'+str(offtcmgrport)+','+offtcmgrip+':'+str(offtcmgrport)+',offmgr,'+\
##                     ',0,0,0,0,0,0,,0,,0,,,'+'no license start failed'+','+\
##                     '0,,0,,0,,0,'
##            log_t.error('1'+msgsend)
            return
    except:
        return
    
    pidf=open('offpidfile','w')
    pidf.write(str(os.getpid()))
    pidf.close()
    global argscomm,argsname,argsvalue,argsdict,conffile
    global offtcmgrname,offtcmgrdomain,offtcmgrip,offtcmgrport
    global tcdomain,tcclusterip,tcport,tcsymbole
    global cachedomain,cacheclusterip,cacheport,cacherouter
    global dbhome,maxthread,manual
    global log_t,log_ip,log_port,myip,myport,sp_domain_name
    global _debug_

##    pdb.set_trace()
    myip=''
    #### linux
    lines = subprocess.Popen('ifconfig | grep "inet addr:"',stdout=subprocess.PIPE,shell=True).stdout.readlines()
    for i in lines:
        if '127.0.0.1' not in i:
##            myip=dict(re.findall(r"(?P<name>.*?):(?P<value>.*?)\n", i))['inet addr']
            inetaddr,myip=re.findall(r"(?P<name>.*?):(?P<value>.*?)\n", i)[0]
            myip=myip.split(' ',1)[0].strip()
            break

##    pdb.set_trace()
    conffile='./offmgrconf.txt'
    argscomm,argsname,argsvalue,argsdict=args.loadargs(conffile)
    all_argsfile='./all_args.pkl'  ##is dict
    if os.path.exists(all_argsfile):
##        os.stat(path)
##        all_argscomm,all_argsname,all_argsvalue,all_argsdict=args.loadargs(all_argsfile)
        fielhandle=open(all_argsfile,'rb')  ##是dict，每个key、value都是string
        all_argsvalue=pickle.load(fielhandle)
        fielhandle.close()
##        if all_argsvalue['type']=='offtcmgr':
        temp_keys=all_argsvalue.keys()
        for i in temp_keys:  ##argsname:
##            globals()[argname]=args.setargsvalue(argvalue,globals()[argname])
            if i in argsdict:
                args.dtupdateargs(argsvalue,argsdict,i,all_argsvalue[i],'')  ##str ????
              ##???  ##需要处理 name和ip配对的问题
        ##offtcmgrname 、offtcmgrclusterip 都是list，顺序号一致，
        ##如果offtcmgrclusterip和myip相同，取offtcmgrname中同一个序号的name
        if 'offtcmgrclusterip' in temp_keys:
            ii=0
            ip_list=all_argsvalue['offtcmgrclusterip'].split(',')
            name_list=all_argsvalue['offtcmgrname'].split(',')
            for i in ip_list:
                if i == myip:
                    args.dtupdateargs(argsvalue,argsdict,'offtcmgrname',\
                                      name_list[ii],'')
                    args.dtupdateargs(argsvalue,argsdict,'offtcmgrip',\
                                      myip,'') #all_argsvalue['offtcmgrip'][ii],'')
                    break
                ii=ii+1
                    
        args.dumpargs(argscomm,argsname,argsvalue,conffile)
        os.remove(all_argsfile)

    _debug_=        args.setargsvalue(argsvalue[argsdict['_debug_']],0)
    offtcmgrname=   args.setargsvalue(argsvalue[argsdict['offtcmgrname']],'offtcmgrname')
    offtcmgrdomain= args.setargsvalue(argsvalue[argsdict['offtcmgrdomain']],'offtcmgrdomain')
    offtcmgrip=     args.setargsvalue(argsvalue[argsdict['offtcmgrip']],[])
    offtcmgrport=   args.setargsvalue(argsvalue[argsdict['offtcmgrport']],8000)
##    bindip=     args.setargsvalue(argsvalue[argsdict['bindip']],'localhost')
##    protype=    'threading' #args.setargsvalue(argsvalue[argsdict['protype']],'forking')

    
    tcdomain=       args.setargsvalue(argsvalue[argsdict['tcdomain']],'tc.vcff.net')
    tcclusterip=    args.setargsvalue(argsvalue[argsdict['tcclusterip']],[])
    tcport=         args.setargsvalue(argsvalue[argsdict['tcport']],8010)
    tcsymbole=      args.setargsvalue(argsvalue[argsdict['tcsymbole']],'&&&tc&&&')  ##转码参数分隔符

    cachedomain=  args.setargsvalue(argsvalue[argsdict['cachedomain']],'cache.vcff.net')
    cacheclusterip=args.setargsvalue(argsvalue[argsdict['cacheclusterip']],[])  #127.0.0.1
    cacheport=  args.setargsvalue(argsvalue[argsdict['cacheport']],8010)
    cacherouter=args.setargsvalue(argsvalue[argsdict['cacherouter']],'transcoded')  #all ##'all' 所有都进cache， 'transcoded' 转码过的文件进cache
    ##

    dbhome=     args.setargsvalue(argsvalue[argsdict['dbhome']],'./db_home') #./db_home
    maxthread=  args.setargsvalue(argsvalue[argsdict['maxthread']],200)
    manual =    args.setargsvalue(argsvalue[argsdict['manual']],1)

    log_ip =    args.setargsvalue(argsvalue[argsdict['log_ip']],'')
    log_port =  args.setargsvalue(argsvalue[argsdict['log_port']],3010)

    sp_domain_name=args.setargsvalue(argsvalue[argsdict['sp_domain_name']],[])

    myport=offtcmgrport

    ##sp_domain_name =['sina','sohu','qiyi','youku','tudou','other_sp']   ## read from pickle or file
    for i in sp_domain_name:
        if os.path.exists(dbhome+'/'+i) == False:
            os.mkdir(dbhome+'/'+i)

    pp=threading.Thread(target=sendlog.Sn_Thread_SendAll,args=(5,1000,  log_ip, log_port))
    pp.start()
    log_t = sendlog.SN_SendLog() #ip='192.168.0.104'
    signal.signal(signal.SIGINT,offtcmgrshutdown)
    signal.signal(signal.SIGTERM,offtcmgrshutdown)
##
##    myip=offtcmgrip
##    if not myip:
##        ## windows
##
####>>> lines = subprocess.Popen('ipconfig /all | grep "IP Address"',stdout=subproce
####ss.PIPE,shell=True).stdout.readlines()
####>>> lines
####['        IP Address. . . . . . . . . . . . : 192.168.0.104\n']
####>>>
####>>> dict(re.findall(r"(?P<name>.*?) : (?P<value>.*?)\n", lines[0]))
####{'        IP Address. . . . . . . . . . . .': '192.168.0.104'}
####        lines = subprocess.Popen('ipconfig /all | grep "IP Address"',\
####                                 stdout=subprocess.PIPE,shell=True).stdout.readlines()
#### linux
##        lines = subprocess.Popen('ifconfig | grep "inet addr:"',stdout=subprocess.PIPE,shell=True).stdout.readlines()
##        for i in lines:
##            if '127.0.0.1' not in i:
##                myip=dict(re.findall(r"(?P<name>.*?):(?P<value>.*?)\r\n", i))['        IP Address. . . . . . . . . . . . ']
##                myip=myip.strip()
####        
####        for i in lines:
####            if '127.0.0.1' not in i:
####                myip=dict(re.findall(r"(?P<name>.*?):(?P<value>.*?) ", i))['inet addr'].strip()
##    myport=offtcmgrport
##
##    business_dict=cPickle.load(open('business_dict.pkl','rb'))
##    sp_domain_name =['sina','sohu','qiyi','youku','tudou','other_sp']   ## read from pickle or file
##    for i in sp_domain_name:
##        if os.path.exists(dbhome+'/'+i) == False:
##            os.mkdir(dbhome+'/'+i)
    
    try:
        socket.request_queue_size=5000
##        threading.stack_size(102400)
        global server
        server = SocketServer.ThreadingTCPServer(('', offtcmgrport), ThreadedTCPRequestHandler)  #SocketServer.ThreadingTCPServer
        ip, port = server.server_address
        # 其它程序将需要下载的url写入此tcp端口，所有收到的url写入一个queue中
        print ('started tcpserver...')
        #server.request_queue_size=5000
        server.daemon_threads=True
        ##server.start()
        ##print("Server loop running in thread:", server_thread.name)threading.current_thread()
        #workth 线程的作用：从queue提取url，然后启动download线程下载此url
        workthid=threading.Thread(target=workth,args=(ip,port))
        workthid.start()
        
##        workthid3=threading.Thread(target=tempthread,args=(ip,port))
##        workthid3.start()
##        pdb.set_trace()
        print("Server loop running in thread:", threading.current_thread())
        msgdict={20:'offtcmgr start ok'}
        msgsend=offtcmgr_logmsg_body(msgdict)
##        msgsend=[1, '%16.4f'%time.time(),3,'22.33.22','127.0.0.1','offmgr.py',\
##             'offtcmgr start ok']
##        log_t.info(msgsend)
##        msgsend=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())+','+\
##                 time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())+','+\
##                 '4,3,'+myip+':'+str(myport)+','+myip+':'+str(myport)+',offmgr,'+\
##                 ',0,0,0,0,0,0,,0,,0,,,'+'offtcmgr start ok'+','+\
##                 '0,,0,,0,,0,'
        log_t.error('1'+msgsend)
        server.serve_forever()
##    except KeyboardInterrupt:
    except :
        print ('^C received or exception, shutting down server')
        server.socket.close()
        taskf.close()
        df.close()
        msgdict={20:'offtcmgr shut down ok'}
        msgsend=offtcmgr_logmsg_body(msgdict)
        log_t.error('1'+msgsend)
        time.sleep(3)
        global shutdown
        shutdown=1
        try:
            os.remove('offpidfile')
        except:pass
        sendlog.sendlogDown()
##        offtcmgrshutdown(1,2)
        
##        server.socket.close()
##        
####        msgsend=[1, '%16.4f'%time.time(),3,'22.33.22','127.0.0.1','offmgr.py',\
####             'offtcmgr shut down']
####        log_t.info(msgsend)
##        
##        sendlog.Glob_shutdown=1
####    pidf=open('offpidfile','w')
####    pidf.write('0')
####    pidf.close()
##    try:
##        os.remove('offpidfile')
##    except:pass


if __name__ == '__main__':
    main()
'''
界面要求：  和一般大下载工具类似
   
   top是标签，有多个，“转码” “监控”  “配置”  “告警”  “日志”，打开下载界面
        
        左侧是“全部任务” “已转码” “正在转码” 浏览已经转码的url，按域名分类文件夹，可折叠打开；
        
        右上侧是转码工具条（可暂停、启动、手工导入下载url list、设置（比如同时转码数量等）等），
            
            右中间是任务列表，正转码和准备转码，有进度，最右侧显示url属性和视频属性
   


测试：
用上面def client发送请求，url用baidu之类的


'''
